//---------------------------------------------------------------------------
/**
 * \class CLineTokenizer
 * \brief This is a tokenizer by line
 * \version 0.1
 *
 * \author Woo-Hyun Kim (woorung@nhncorp.com, woorung@gmail.com)
 * \author http://blog.naver.com/wisereign .
 *
 * \date 2007/10/08
 */
//---------------------------------------------------------------------------
#include "CLineTokenizer.h"
//----------------------------------------------------------------------
CLineTokenizer::CLineTokenizer() 
{
	use_file_stream = true;

	block_stream = NULL;

	kpos = 1;
	delim = '\t';
	eol = '\n';
	limit = MAX_LINE_LIMIT;
}
//----------------------------------------------------------------------
CLineTokenizer::~CLineTokenizer() 
{
	delete block_stream;
}
//----------------------------------------------------------------------
void CLineTokenizer::assign(long block, long blocks) 
{
	if(use_file_stream) {
		block_stream->assignBlock(block, blocks);
	}
}
//----------------------------------------------------------------------
void CLineTokenizer::input(char *file) 
{
	use_file_stream = true;

	block_stream = new CBlockStream(file);

	if(use_file_stream) block_stream->resize(1024);

	kpos = 1;
	delim = '\t';
	eol = '\n';
	limit = MAX_LINE_LIMIT;
}
//----------------------------------------------------------------------
void CLineTokenizer::input(streambuf *stdbuf) 
{
	use_file_stream = false;

	block_stream = new CBlockStream(stdbuf);

	// no use due to error
	if(use_file_stream) block_stream->resize(1024);

	kpos = 1;
	delim = '\t';
	eol = '\n';
	limit = MAX_LINE_LIMIT;
}
//----------------------------------------------------------------------
void CLineTokenizer::input(stringbuf *strbuf) 
{
	use_file_stream = false;

	block_stream = new CBlockStream(strbuf);

	// no use due to error
	if(use_file_stream) block_stream->resize(1024);

	kpos = 1;
	delim = '\t';
	eol = '\n';
	limit = MAX_LINE_LIMIT;
}
//----------------------------------------------------------------------
void CLineTokenizer::output() 
{
}
//----------------------------------------------------------------------
void CLineTokenizer::format(int kpos) 
{
	this->kpos = kpos;
}
//----------------------------------------------------------------------
void CLineTokenizer::format(int kpos, vector<int> &ppos) 
{
	for(int i=0;i<ppos.size();i++) this->ppos.push_back(ppos[i]);

	this->kpos = kpos;
}
//----------------------------------------------------------------------
void CLineTokenizer::filter(char delim, char eol, long limit) 
{
	this->delim = delim;
	this->eol = eol;
	this->limit = limit;
}
//----------------------------------------------------------------------
int CLineTokenizer::getToken(vector< vector<string> > &token)
{
	if(block_stream->getline(token, delim, limit)!=0) return 1;
	else return 0;
}
//----------------------------------------------------------------------




/*
//######################################################################
//			Thread-based Tokenizer
//######################################################################
//----------------------------------------------------------------------
// must be overriden
//----------------------------------------------------------------------
int CLineTokenizer::tokenizer(vector< vector<string> > &kv_vectors)
{
	int key = kpos-1;

	// only print the selected items 
	if(ppos.size()>0) {
		for(int i=0;i<kv_vectors.size();i++) {
			if(kv_vectors[i].size()<=key) continue;

			cout << "#[" << kv_vectors[i][key] << "]" << endl;

			for(int j=0;j<ppos.size();j++) cout << kv_vectors[i][ppos[j]-1] << delim;
			cout << endl;
		}
	}
	// print all the items
	else {
		for(int i=0;i<kv_vectors.size();i++) {
			if(kv_vectors[i].size()<=key) continue;

			cout << "#[" << kv_vectors[i][key] << "]" << endl;

			for(int j=0;j<kv_vectors[i].size();j++) cout << kv_vectors[i][j] << delim;
			cout << endl;
		}
	}
}
//----------------------------------------------------------------------
// must be overriden
//----------------------------------------------------------------------
int CLineTokenizer::tokenizer(ofstream &fout, vector< vector<string> > &kv_vectors)
{
	int key = kpos-1;

	// only print the selected items 
	if(ppos.size()>0) {
		for(int i=0;i<kv_vectors.size();i++) {
			if(kv_vectors[i].size()<=key) continue;

			fout << "#[" << kv_vectors[i][key] << "]" << endl;

			for(int j=0;j<ppos.size();j++) fout << kv_vectors[i][ppos[j]-1] << delim;
			fout << endl;
		}
	}
	// print all the items
	else {
		for(int i=0;i<kv_vectors.size();i++) {
			if(kv_vectors[i].size()<=key) continue;

			fout << "#[" << kv_vectors[i][key] << "]" << endl;

			for(int j=0;j<kv_vectors[i].size();j++) fout << kv_vectors[i][j] << delim;
			fout << endl;
		}
	}
}
//----------------------------------------------------------------------
void CLineTokenizer::run() 
{
	if(use_file_stream) block_stream->resize(1024);

#ifdef USE_FILE
	char uniq_file[100];
	string thread_id = getName();
	sprintf(uniq_file, "xxx-%d-%s", getpid(), (char *)thread_id.c_str());
	ofstream fout(uniq_file, ios::app|ios::out);
#endif

	vector< vector<string> > kv_vectors;
	while(block_stream->getline(kv_vectors, delim, limit)!=0) {
#ifdef USE_FILE
		// print to file
		tokenizer(fout, kv_vectors);
#else
		// print to stdout 
		tokenizer(kv_vectors);
#endif
		kv_vectors.clear();
	}

#ifdef USE_FILE
	fout.close();
#endif
}
//----------------------------------------------------------------------
*/
